package com.wuda.lucene;

import java.io.IOException;
import java.io.Reader;

import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;

import com.wuda.cfg.Configure;

/**
 * Tokenizes input text into sentences.
 * <p>
 * The output tokens can then be broken into words with {@link GTTokenFilter}
 * </p>
 * 
 */
public final class GTSentenceTokenizer extends Tokenizer {

	/**
	 * End of sentence punctuation: 。，！？；,!?;
	 */
	private final static String PUNCTION = Configure.PUNCTION;

	private final StringBuilder buffer = new StringBuilder();

	private int tokenStart = 0, tokenEnd = 0;

	private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
	private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
	private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);

	public GTSentenceTokenizer(Reader reader) {
		super(reader);
	}

	public GTSentenceTokenizer(AttributeFactory factory, Reader reader) {
		super(factory, reader);
	}

	@Override
	public boolean incrementToken() throws IOException {
		clearAttributes();
		buffer.setLength(0);
		int ci;
		char ch;
		tokenStart = tokenEnd;
		ci = input.read();
		ch = (char) ci;

		while (true) {
			if (ci == -1) {
				break;
			} else if (PUNCTION.indexOf(ch) != -1) {
				// End of a sentence
				// buffer.append(ch); 不需要这个标点符号
				tokenEnd++;
				break;
			} else {
				buffer.append(ch);
				tokenEnd++;
				ci = input.read();
				ch = (char) ci;
			}
		}
		if (buffer.length() == 0)
			return false;
		else {
			termAtt.setEmpty().append(buffer);
			offsetAtt.setOffset(correctOffset(tokenStart),
					correctOffset(tokenEnd));
			typeAtt.setType("sentence");
			return true;
		}
	}

	@Override
	public void reset() throws IOException {
		tokenStart = tokenEnd = 0;
	}

	@Override
	public void end() {
		// set final offset
		final int finalOffset = correctOffset(tokenEnd);
		offsetAtt.setOffset(finalOffset, finalOffset);
	}
}
